package com.spider.ask120.webmagic.task;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.jdiy.core.Rs;
import org.jdiy.json.JsonArray;
import org.jdiy.json.JsonObject;
import org.jdiy.json.JsonUtil;

import com.spider.ask120.service.ArticleService;
import com.spider.ask120.service.ChannelArticleInfoService;
import com.spider.ask120.service.ChannelService;
import com.spider.ask120.webmagic.Base;
import com.spider.utils.HttpClientUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康经验正文
 * @author Jxw
 *
 */
public class Jingyan extends Base implements PageProcessor{
	private static Logger logger = Logger.getLogger(Jingyan.class);
	static String jkjyUrl = "http://m.120ask.com/jingyan/";
	static String listUrl = "http://m.120ask.com/jingyan/list/\\w+"; 
	
	public static void main(String[] args) {
//		Function fun = new Function();
//		String spider_scheduler_path = fun.getProValue(JINGYAN_SCHEDULER_PATH);
//		FileCacheQueueScheduler scheduler = new FileCacheQueueScheduler(spider_scheduler_path);
//		try {
//			List<Rs> channels = ChannelService.getChannelLowCatalog();
//			for(Rs channel : channels){
//				String id = channel.get("id");
//				String str_id = channel.get("str_id");
//				Spider s = Spider.create(new Jingyan()).addUrl("http://m.120ask.com/jingyan/list/"+str_id+"?cata_id="+id).scheduler(scheduler); 
//				SpiderMonitor.instance().register(s);
//				s.thread(1).start();
//			}
//		} catch (Exception e) {
//			logger.info(e);
//		}
		Spider.create(new Jingyan()).addUrl("http://m.120ask.com/jingyan/list/1kwv80sksk?cata_id=2284").thread(1).run();
    }
	
 
	public void process(Page page) {
		String pic_dir = fun.getProValue(JINGYAN_PIC_FILE_PATH);
		if(page.getUrl().regex(listUrl).match()) {//不包括.html后缀  //+"$"  
			//获取?cata_id= 后面的栏目ID
			String path = page.getUrl().toString();
			String cata_id = path.substring(path.indexOf("?cata_id=")+9);
			
			Rs ca = ChannelArticleInfoService.getInfoByChId(Integer.parseInt(cata_id));
			Integer record_str_id = ca.getInt(ARTICLE_STR_ID);
			Integer bigger_str_id = record_str_id;
			
			//无下级栏目，开始加载分页数据列表，获取正文URL
			Map<String, String> params = new HashMap<String, String>();
			params.put("ajax", "1");
			//从第一页开始取， 只能拿到HTML页面内容，取不到json格式的数据，所以跳过第一页；
			int pageIndex = 2;	
			Integer count = 1;
			list:
			while(true){ 
				params.put("page", pageIndex+"");
				String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
				if(listJson != null && !listJson.isEmpty()){  
					try {
						JsonObject rstObj = (JsonObject) JsonUtil.parse(listJson); //{"code":200,"list":[]}
						if(rstObj != null && rstObj.get("code") != null && "200".equals(rstObj.get("code").toString()) 
										&& rstObj.get("list") != null && !"[]".equals(rstObj.get("list").toString())){
							JsonArray objAry = (JsonArray) JsonUtil.parse(rstObj.get("list").toString());
							if(objAry.size() > 0){ 
								for(int i = 0; i < objAry.size(); i++){
									JsonObject obj = (JsonObject) objAry.get(i); 
//									//System.out.println(obj.toString());
									String str_id = obj.get("id_str").toString();
									str_id = str_id.substring(0, str_id.indexOf("."));
									
									//如果文章Id比数据库中存储的最新文章Id小 跳出循环
									Integer current_article_str_id = Integer.parseInt(obj.get("id").toString()); 
									if (record_str_id >= current_article_str_id) {
										break list;
									}
									
									//判断文章是否已存在
									Rs art = ArticleService.getArticleByStrId(cata_id, obj.get("title").toString(), obj.get("id").toString());
									String a_id = "";
									if (art.isNull()) {
										Rs article = new Rs("article");  
										article.set("str_id", obj.get("id").toString());
										article.set("channel_id", cata_id);
										article.set("title", obj.get("title").toString());
										article.set("abst", obj.get("content").toString());
										//下载图片到本地并返回图片地址
										String pic_path = fun.downloadPic(obj.get("pic").toString(),pic_dir+"list/");
										article.set("pic", pic_path);
										ArticleService.saveArticle(article);
										a_id = article.get("id");
									}else {
										a_id = art.get("id");
									}
									
									//将文章正文url加入请求队列
									Map<String, Object> extras = new HashMap<String, Object>();
									extras.put("article_id", a_id); 
									Request request = new Request("http://m.120ask.com/jingyan/"+obj.get("id_str").toString()+".html");
									request.setExtras(extras);
									page.addTargetRequest(request);
									
									//记录最新文章str_id
									if (count == 1) {
										bigger_str_id = current_article_str_id;
										count = 2;
									}
								}
							}else{ 
								break;
							}
						}else{ 
							break;
						}
					} catch (Exception e) { 
						e.printStackTrace();
					}
				}
				
				pageIndex ++;
				//只取一页的数据
//				break;
			}
			ca.set(ARTICLE_STR_ID, bigger_str_id);
			ChannelArticleInfoService.save(ca);
		}else if(page.getUrl().regex(jkjyUrl+"\\w+.html").match()) {
			String title = page.getHtml().xpath("//div[@class='g-jinyan-topic']/h3/text()").get();
			String create_time = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").regex("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}").get();
			String share = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[1]/text()").toString();
			share = share.substring(share.indexOf("由")+1, share.indexOf("分享")).trim(); 
			String abst = page.getHtml().xpath("//div[@class='g-jinyan-cont']/p/text()").get();			//介绍
			String remind = page.getHtml().xpath("//div[@class='g-jinyan-conf']/p[2]/html()").get();	//注意事项
			List<String> contStep = page.getHtml().xpath("//div[@class='d1 img']/dl/dd/html()").all();		//步骤方法
			List<String> keywords = page.getHtml().xpath("//div[@class='g-jinyan-topic']/p[2]/a/text()").all();
//			contStep = replacePicPath(contStep,pic_dir);

			String article_id = page.getRequest().getExtra("article_id").toString();
			Rs article = ArticleService.getArticleById(article_id);
			if(article != null && !article.isNull()){ 
				article.set("title", title);
				article.set("create_time", create_time);
				article.set("source", share); 
				
				if(keywords.size() > 0){ 
					String keyword = "";
					for(String kw : keywords){
						keyword += keyword.isEmpty() ? kw : ","+kw;
					}
					article.set("keyword", keyword); 
				} 
				JsonArray stepArr = new JsonArray();
				for(int i = 0; i < contStep.size(); i++){
					JsonObject step = new JsonObject();
					step.set("step_index", (i+1));
					//System.out.println(contStep.get(i));
					//System.out.println(replacePicPath(contStep.get(i),pic_dir));
					step.set("step_content", replacePicPath(contStep.get(i),pic_dir)); 
					stepArr.add(step);
				}
				
				JsonObject content = new JsonObject();
				content.set("abst", abst);
				content.set("remind", remind); 
				content.set("steps", stepArr);
				
				article.set("content", content.toString());
				ArticleService.updateArticle(article);
			}
			  
 
		} 
	}

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}
	/**
	 * 替换文章中的url为本地url
	 * @param contStep 
	 * @return 本地保存后地址
	 */
	@SuppressWarnings("unused")
	private List<String> replacePicPath(List<String> contStep,String pic_dir){
		List<String> list = new ArrayList<String>();
		for (int m = 0; m < contStep.size(); m++) {
			String str = contStep.get(m);
			int i = str.indexOf("src=\"");
			if (-1 != i) {
				i += 5;
				int j = str.indexOf("\" alt");
				String pic_url = str.substring(i, j);
				//下载图片到本地并返回图片地址
				String pic_path = fun.downloadPic(pic_url, pic_dir + "jkjy/"+date+"article/");
				String replace = str.replace(pic_url, pic_path);
				list.add(replace);
			}else {
				list.add(str);
			} 
		}
		return list;
	}
	/**
	 * 替换文章中的url为本地url
	 * @param contStep 
	 * @return 本地保存后地址
	 */
	private String replacePicPath(String str,String pic_dir){
		int i = str.indexOf("src=\"");
		if (-1 != i) {
			i += 5;
			int j = str.indexOf("\" alt");
			String pic_url = str.substring(i, j);
			//下载图片到本地并返回图片地址
			String pic_path = fun.downloadPic(pic_url, pic_dir + "jkjy/"+date+"/article/");
			String replace = str.replace(pic_url, pic_path);
			return replace;
		}else {
			return str;
		} 
	}
	public static void runJingyan(){
		try {
			List<String> list = new ArrayList<String>();
			List<Rs> channels = ChannelService.getChannelLowCatalog();
			for(Rs channel : channels){
				String id = channel.get("id");
				String str_id = channel.get("str_id");
				list.add("http://m.120ask.com/jingyan/list/"+str_id+"?cata_id="+id);
			}
			int size=list.size();  
	        String[] array=new String[size];  
	        for(int i=0;i<list.size();i++){  
	            array[i]=(String)list.get(i);  
	        } 
			Spider s = Spider.create(new Jingyan()).addUrl(array); 
			s.thread(1).start();
		} catch (Exception e) {
			logger.info(e);
		}
    }
}
